package cetcbigdata.da.service.zhejiang.qitaqingdan;

import cetcbigdata.da.base.Base;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.alibaba.datax.common.plugin.RecordSender;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class SpiderZheJiang extends Base {
    private static String baseRegion = "浙江省";

    private static String baseModule = "服务事项-服务清单-同办清单";

    @Override
    public void getData(RecordSender recordSender, List<Object> fieldsList) {
        List<JSONObject> data = new ArrayList<>();
        JSONObject item1 = new JSONObject();
        JSONObject item2 = new JSONObject();
        JSONObject item3 = new JSONObject();
        JSONObject params1 = new JSONObject();
        JSONObject params2 = new JSONObject();
        JSONObject params3 = new JSONObject();
        String time3 = String.valueOf(System.currentTimeMillis());

        item1.put("name", "服务清单-权责清单");
        item1.put("url", "https://www.zjzwfw.gov.cn/zjservice/matter/list/qzqdlist.do");
        params1.put("webId", "");
        params1.put("dept", "");
        params1.put("bigCategory", "");
        params1.put("pagesize", 20);
        params1.put("pageno", 1);
        params1.put("_", time3);
        item1.put("params", params1);

        item2.put("name", "服务清单-跑零次清单");
        item2.put("url", "https://www.zjzwfw.gov.cn/zjservice/matter/list/list.do");
        params2.put("jurisCode", "");
        params2.put("pagesize", 20);
        params2.put("pageno", 1);
        params2.put("categoryType", 1);
        params2.put("dept", "");
        params2.put("_", time3);
        item2.put("params", params2);


        item3.put("name", "服务清单-最多跑一次清单");
        item3.put("url", "https://www.zjzwfw.gov.cn/zjservice/item/runonce/list.do");
        params3.put("webId", "");
        params3.put("dept", "");
        params3.put("istown", "");
        params3.put("groupcode", "");
        params3.put("pagesize", 20);
        params3.put("pageno", 1);
        params3.put("undefined", 1);
        params3.put("_", time3);
        item3.put("params", params3);

        data.add(item1);
        data.add(item2);
        data.add(item3);
        for(JSONObject topic : data){
            this.parseListPage(recordSender, fieldsList, topic);
        }
    }

    @Override
    public JSONObject getDataDemo() {
        try {
            return processResult("xxx", "xxx", "xxx", "xxx", new JSONObject(), "xxx", "xxx");
        }catch (Exception e){
            e.printStackTrace();
        }
        return null;
    }

    private void parseListPage(RecordSender recordSender, List<Object> fieldsList, JSONObject topic){
        try {
            for(JSONObject regionItem : getRegionList()){
                    getPageList(recordSender, fieldsList, regionItem, topic);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /***
     * 获取主题信息： 生育收养、 户籍办理 等
     * @return List<JSONObject>
     */
    private static void getPageList(RecordSender recordSender, List<Object> fieldsList,
                                    JSONObject regionItem, JSONObject topic){
        // 当没有数据返回时，网站中会出现 div[@id='nodata'] 元素，则停止翻页
        JSONObject headers = UtilZheJiang.getHeaders();
        JSONObject params = topic.getJSONObject("params");
        while (true){
            if(params.containsKey("jurisCode")){
                params.put("jurisCode", regionItem.getString("code"));
            }else {
                params.put("webId", regionItem.getInteger("id"));
            }

            String listResponse =  getReq(topic.getString("url"), headers, params);
            String regex = "var _0x4818=([\\s\\S]*?);";
            Pattern pattern = Pattern.compile(regex);
            Matcher matcher = pattern.matcher(listResponse);
            String listUrl = topic.getString("url");
            // 判断是否需要更新cookie参数
            if(matcher.find()){
                // 需要更新
                regex = "var arg1=([\\s\\S]*?);";
                pattern = Pattern.compile(regex);
                matcher = pattern.matcher(listResponse);
                String acw_sc__v2 = null;
                if (matcher.find()) {
                    String arg1 = matcher.group();
                    arg1 = arg1.replace("var arg1='", "");
                    arg1 = arg1.replace("';", "");
                    acw_sc__v2 = cetcbigdata.da.personal_service.zhejiang.Js.getEncode(arg1);
                }
                // 更新cookie
                headers.put("Cookie", "acw_sc__v2=" + acw_sc__v2);
                // 更新cookie后，重新请求
                listResponse =  getReq(listUrl,
                        headers, params);
            }

            parseListPage(listUrl, listResponse, recordSender, fieldsList, regionItem, topic);

            // 页面中出现， 很抱歉，暂时没有您所需的记录！ 标识没有数据，停止翻页
            JXDocument jxDocument = new JXDocument(listResponse);
            if(jxDocument.sel("//div[@class='wdzj_norecord']").size() == 1){
                break;
            }

            // 翻页处理
            JSONObject parmas = topic.getJSONObject("params");
            parmas.put("pageno", parmas.getInteger("pageno") + 1);
            parmas.put("_",  System.currentTimeMillis());
            topic.put("params", parmas);
        }
    }

    private static void parseListPage(String listUrl, String listResponse, RecordSender recordSender, List<Object> fieldsList,
                                      JSONObject regionItem, JSONObject topic){
        JXDocument jxDocument = new JXDocument(listResponse);
        // 处理没有子选项
        List<Object> items = jxDocument.sel("//div[@class='body_right']/div[@class='rdbs_list childItem']");
        for (Object o : items){
            // 声明定义列表页每一项数据
            JSONObject content = new JSONObject();
            String title  =  ((Element) o).child(0).child(0).child(0).text();
            String url  = ((Element) o).child(0).child(0).child(0).attr("onclick");
            url = getSting(url);
            url = "https://www.zjzwfw.gov.cn/zjservice/item/detail/index.do?localInnerCode=" + url;
            content.put("名称", title);
            content.put("地址", url);
            printDataCount(50);
            processItem(
                    processResult(
                            listUrl, listResponse, url, "",
                            content,
                            baseRegion + "-" + regionItem.getString("name"),
                            baseModule + "-" + topic.getString("name")),
                    recordSender,
                    fieldsList
            );
        }


        // 处理有子选项
        items = jxDocument.sel("//div[@class='rdbs_list']");
        for (Object o : items){
            //
            if(((Element) o).parent().className().equals("rdbs_list_subtitmain")){
                continue;
            }
            JSONObject out = new JSONObject();
            List<JSONObject> xuanxiang1 = new ArrayList<>();

            String fisrtTitle  =  ((Element) o).child(0).child(0).text();
            Elements children  =  ((Element) o).child(1).children();
            for(Element e : children){
                if(e.tagName().equals("script")){
                    continue;
                }
                String subtitle = null;
                String title = null;
                if(e.className().equals("rdbs_list_subtitmain childItem")){
                    JSONObject item2 = new JSONObject();
                    // 子选项
                    subtitle  = e.child(0).child(0).text();

                    String url  =  e.child(0).child(0).attr("onclick");
                    url = getSting(url);
                    url = "https://www.zjzwfw.gov.cn/zjservice/item/detail/index.do?localInnerCode=" + url;
                    item2.put("名称", subtitle);
                    item2.put("地址", url);
                    xuanxiang1.add(item2);
                }

                else if(e.className().equals("rdbs_list")){
                    JSONObject subsubItem = new JSONObject();
                    List<JSONObject> xuanxiang2 = new ArrayList<>();
                    subtitle = e.child(0).child(0).text();
                    // 孙选项
                    for(Element ee :  e.child(1).children()){
                        JSONObject item3 = new JSONObject();
                        title  =  ee.child(0).child(0).text();
                        String url  =  e.child(0).child(0).attr("onclick");
                        url = getSting(url);
                        url = "https://www.zjzwfw.gov.cn/zjservice/item/detail/index.do?localInnerCode=" + url;
                        item3.put("名称", title);
                        item3.put("地址", url);
                        xuanxiang2.add(item3);
                    }
                    subsubItem.put("名称", subtitle);
                    subsubItem.put("选项", xuanxiang2);
                    xuanxiang1.add(subsubItem);
                }
            }

            out.put("名称", fisrtTitle);
            out.put("选项", xuanxiang1);
            printDataCount(50);
            processItem(
                    processResult(
                            listUrl, listResponse, "", "",
                            out,
                            baseRegion + "-" + regionItem.getString("name"),
                            baseModule + "-" + topic.getString("name")),
                    recordSender,
                    fieldsList
            );
        }
    }

    private static String getSting(String url){
        String regex = "articlePage\\('([\\s\\S]*?)',";
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(url);
        String data = null;
        while (matcher.find()) {
            data = matcher.group();
            data = data.replace("articlePage('", "").replace("',", "");
            return data;
        }
        return "";
    }


    private static List<JSONObject> getRegionList(){
        String url = "https://zjjcmspublic.oss-cn-hangzhou-zwynet-d01-a.internet.cloud.zj.gov.cn/jcms_files/jcms1/web1/site/script/zjzwfw-2019/new_json2.js";
        JSONObject headers = new JSONObject();
        headers.put("Host", "zjjcmspublic.oss-cn-hangzhou-zwynet-d01-a.internet.cloud.zj.gov.cn");
        headers.put("Referer", "https://www.zjzwfw.gov.cn/");
        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36");
        String response = null;
        try {
            String responseOrg = getReq(url, headers, UtilZheJiang.getParams());
            response = new String(responseOrg.getBytes("iso-8859-1"), "utf-8");
        }catch (Exception e){
            e.printStackTrace();
        }
        response = response.replace("var jsonwebsite=", "");
        JSONArray dataList =  ( (JSONObject) JSONObject.parseArray(response).get(0)).getJSONArray("childs");
        List<JSONObject> out = new ArrayList<>();
        // 遍历市
        for(Object o : dataList){
            JSONObject shiItem = (JSONObject) o;
            // 遍历区
            JSONArray childs = shiItem.getJSONArray("childs");
            for(Object oo : childs){
                JSONObject quItem = (JSONObject) oo;
                JSONObject outItem = new JSONObject();
                outItem.put("code", quItem.getString("code"));
                outItem.put("id", quItem.getString("id"));
                outItem.put("name", shiItem.getString("name") + "-" + quItem.getString("name"));
                out.add(outItem);
            }
        }
        return out;
    }

    public static void main(String[] args) {
        SpiderZheJiang spiderZheJiang = new SpiderZheJiang();
        spiderZheJiang.getData(null, new ArrayList<>());
    }
}
